#!/usr/bin/env python
# coding=utf-8
# __author__ = 'Yunchao Ling'

from retrying import retry
import cStringIO
import pycurl
# import requests
import datetime
import time
import os
import sys

NOW_DATE = datetime.datetime.now().strftime("%Y%m%d")


def checkSpace(dev):
    import commands
    import re

    space = 10
    line = commands.getoutput("df -B G -P")
    splitline = line.split("\n")
    for item in splitline:
        if item.startswith(dev):
            item = item.rstrip()
            regex = "^.*? *\d*G *\d*G *(\d*)G *\d*% *.*$"
            match = re.match(regex, item)
            if match:
                result = match.group(1)
                space = int(result)
                break
    return space


def Verify_Meta_XML(filename):
    import commands

    cmd = "grep '</pmc-articleset>' " + filename
    result = commands.getoutput(cmd)
    if result.find("</pmc-articleset>") != -1:
        # print filename + " is valid."
        return True
    elif result.find("No such file or directory") != -1:
        # print "Cannot find "+filename+". Try to redownload..."
        return False
    else:
        # print filename + " is invalid. Try to redownload..."
        return False


@retry
def Get_PMC_Meta(filename):
    ids = ""
    infile = open(filename, "r")
    for line in infile:
        line = line.rstrip()
        splitline = line.split("\t")
        ids += splitline[0] + ","
    infile.close()
    ids = ids[:-1]
    taskid = filename.split(".")[-2]
    filetype = filename.split(".")[-5]
    xml_name = "pmcmeta." + filetype + "." + NOW_DATE + "." + taskid + ".xml"
    if os.path.exists(xml_name):
        os.remove(xml_name)
    url = 'https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi'
    buf = cStringIO.StringIO()
    c = pycurl.Curl()
    c.setopt(pycurl.POST, 1)
    c.setopt(c.URL, url)
    c.setopt(pycurl.HTTPPOST, [('db', 'pmc'), ('id', ids), ('retmode', 'xml')])
    c.setopt(c.WRITEFUNCTION, buf.write)
    c.perform()
    outfile = open(xml_name, "w")
    outfile.write(buf.getvalue())
    outfile.close()
    buf.close()
    c.close()
    print "Verifying " + xml_name + " ... ",
    if not Verify_Meta_XML(xml_name):
        print("invalid. Try to redownload...")
        raise IOError("Invalid xml.")
    else:
        print("OK.")
        print(xml_name + " download complete.")


# @retry
# def Get_PMC_Meta(filename):
#     ids = ""
#     infile = open(filename, "r")
#     for line in infile:
#         line = line.rstrip()
#         splitline = line.split("\t")
#         ids += splitline[0] + ","
#     infile.close()
#     ids = ids[:-1]
#     taskid = filename.split(".")[-2]
#     filetype = filename.split(".")[-5]
#     xml_name = "pmcmeta." + filetype + "." + NOW_DATE + "." + taskid + ".xml"
#     if os.path.exists(xml_name):
#         os.remove(xml_name)
#     url = 'https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi'
#     payload = {"db": "pmc", "id": ids, "retmode": "xml"}
#     r = requests.get(url, params=payload)
#     with open(xml_name, 'wb') as fd:
#         for chunk in r.iter_content(1024 * 1024):
#             fd.write(chunk)
#     fd.close()
#     print "Verifying " + xml_name + " ... ",
#     if not Verify_Meta_XML(xml_name):
#         print("invalid. Try to redownload...")
#         raise IOError("Invalid xml.")
#     else:
#         print("OK.")
#         print(xml_name + " download complete.")


if __name__ == '__main__':
    print("Loading date, current given date is ... "),
    time.sleep(1)
    if len(sys.argv) >= 2:
        NOW_DATE = sys.argv[1]
    print(NOW_DATE + ".")
    time.sleep(1)

    print("Spliting task file ... "),
    if not (os.path.exists('pmcid_todo.new.' + NOW_DATE + '.txt') or os.path.exists(
                    'pmcid_todo.update.' + NOW_DATE + '.txt')):
        print('Cannot find todo list files, program abort.')
        sys.exit(0)
    if os.path.exists('pmcid_todo.new.' + NOW_DATE + '.txt'):
        os.system("python SplitList.py pmcid_todo.new." + NOW_DATE + ".txt 500")
    if os.path.exists('pmcid_todo.update.' + NOW_DATE + '.txt'):
        os.system("python SplitList.py pmcid_todo.update." + NOW_DATE + ".txt 500")
    time.sleep(1)
    print("complete.")
    time.sleep(1)

    print("Loading splitted task files ... "),
    filelist = os.listdir(".")
    time.sleep(1)
    print("complete.")
    time.sleep(1)

    for filename in filelist:
        if filename.endswith(".task"):
            while checkSpace("/dev/xvda") < 5:
                time.sleep(30)
            print("Start downloading task " + filename + " ... ")
            Get_PMC_Meta(filename)
            time.sleep(1)

            print("Deleting task file ... "),
            if os.path.exists(filename):
                os.remove(filename)
            time.sleep(1)
            print("complete.")
            time.sleep(1)

            print("Task " + filename + " complete.")
            time.sleep(1)

    print("Compressing xml files ... "),
    os.system("tar -czf pmcmeta." + NOW_DATE + ".tar.gz pmcmeta.*." + NOW_DATE + ".*.xml --remove-files")
    time.sleep(1)
    print("complete.")
    time.sleep(1)

    print("Calculating md5 ... "),
    os.system("md5sum pmcmeta." + NOW_DATE + ".tar.gz >pmcmeta." + NOW_DATE + ".tar.gz.md5")
    time.sleep(1)
    print("complete.")
    time.sleep(1)

    print("Deleting todo list file ... "),
    if os.path.exists('pmcid_todo.new.' + NOW_DATE + '.txt'):
        os.remove('pmcid_todo.new.' + NOW_DATE + '.txt')
    if os.path.exists('pmcid_todo.update.' + NOW_DATE + '.txt'):
        os.remove('pmcid_todo.update.' + NOW_DATE + '.txt')
    time.sleep(1)
    print("complete.")
    time.sleep(1)

    print("Setting complete symbol ... "),
    os.system("touch all.complete." + NOW_DATE)
    time.sleep(1)
    print("complete.")
    time.sleep(1)

    print("All tasks completed.")
